#!/usr/bin/env python
#-*- coding:utf-8 -*-
import sys,os,re
import threading
from wwpy.util.Time import Time
from wwpy.util.Url import Url
from wwpy.util.Logger import logger
from wwpy.wuba.data.DataDict import DisplayLocalDict,DisplayCateDict
from wwpy.db.DBI import DBI,dbc
from wwpy.util.Wrapper import looper


class MlogTelphone(object):
    def __init__(self,date=None,input='.',output='.',sep=',',db='db_pre_ads'):
        self.date=date or Time.yesterday('')
        self.input=input
        self.output=output
        self.sep=sep
        self.db=db
        self.line_regex=re.compile(r'/ga/url/\?path=(?P<path>[^&]+)')
        self.path_regex=re.compile(r'/telphone/(?P<page_type>clicklist|clickdetail)/?')
        self.detail_others=dict([(sth,idx) for idx,sth in enumerate('phone,ico,num'.split(','))])
        
    @property
    def hosts(self):
        return '10.5.18.12 10.5.18.13 10.5.18.9 10.5.20.37 10.5.20.38 10.5.20.39 10.5.20.40'.split(' ')
        
    def input_file(self,host):
        return self.input+os.sep+'mlog_path_%s_%s.log' % (self.date,host)
        
    def get_log(self,host):
        log=self.input_file(host)
        url=Url('http://10.5.12.167:50075/streamFile/dsap/rawdata/mlog-list/%s/%s_m.58.com_access_log.%s.txt' % (self.date,host,Time.date_format(self.date)))
        if not os.path.exists(log):
            logger.info("Download log: %s from %s" % (log,url.url))
            url.download(log,grep=r'/ga/url/\?path=(?:[^&]+)')
        
    def get_logs(self,use_threads=False):
        hosts=self.hosts
        if use_threads:
            threads=[]
            for host in hosts:
                thread=threading.Thread(target=self.get_log,args=(host))
                thread.setDaemon(True)
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
        else:
            for host in hosts:
                self.get_log(host)
            
    def process_log(self,log):
        logger.info("Process log: %s" % log)
        f=open(log,'rb')
        for line in f:
            fields=self.process_line(line)
            if fields is not None:
                yield fields
        
    def process_logs(self):
        logs=map(self.input_file,self.hosts)
        for log in logs:
            if not os.path.exists(log):
                continue
            for fields in self.process_log(log):
                yield [self.date]+list(fields)
                
    def map(self):
        return self.process_logs()
                
    def scan(self,func=None,*args,**kwargs):
        func=func or self.map
        results=func(*args,**kwargs)
        for fields in results:
            print self.sep.join(str(field) for field in fields)
    
    @property
    def file_dict(self):
        return {
            self.map_file:self.map,
            self.reduce_file:self.reduce,
        }
    
    def store(self,file=None,*args,**kwargs):
        file=file or self.reduce_file
        func=self.file_dict.get(file)
        results=func(*args,**kwargs)
        logger.info("Store file: %s" % file)
        f=open(file,'wb')
        for fields in results:
            f.write(self.sep.join(str(field) for field in fields)+'\n')
        f.close()
        
    @property
    def map_file(self):
        return self.output+os.sep+'mlog_telphone_map_%s.txt' % (self.date)
        
    @property
    def reduce_file(self):
        return self.output+os.sep+'mlog_telphone_reduce_%s.txt' % self.date
    
    def process_line(self,line):
        matched=self.line_regex.search(line[:-1])
        if matched:
            return self.process_path(matched.group('path'))
        return None
    
    def process_path(self,path):
        matched=self.path_regex.search(path)
        if matched:
            if not path.endswith('/'):
                path+='/'
            page_type=matched.group('page_type')
            if page_type=='clicklist':
                return self.process_list(path)
            elif page_type=='clickdetail':
                return self.process_detail(path)
        return None
    
    @property
    def city_dict(self):
        citys={}
        o=DisplayLocalDict()
        rows=o.select(fields=('local_id','list_name','city1','city2'))
        ids,names=dict(),dict()
        for row in rows:
            local_id,list_name,city1,city2=row
            ids[local_id]=(city1,city2)
            names[list_name]=(city1,city2)
        citys['id']=ids
        citys['name']=names
        return citys
    
    @property
    def cate_dict(self):
        cates={}
        o=DisplayCateDict()
        rows=o.select(fields=('list_name','cate1','cate2','cate3'))
        for row in rows:
            list_name,cate1,cate2,cate3=row
            cates[list_name]=(cate1,cate2,cate3)
        return cates
    
    def process_list(self,path):
        #path=/city_id/cate_name/telphone/clicklist/(ad_id/ad_pos/)?
        page_type=0
        fields=path.split('/')
        city_id,cate_name=fields[1],fields[2]
        if fields[5]:
            ad_id,ad_pos=fields[5],fields[6]
        else:
            ad_id,ad_pos=-1,-1
        return page_type,city_id,cate_name,ad_id,ad_pos
    
    def process_detail(self,path):
        #path=/cate_name/big_cate_name/telphone/clickdetail/((ad_id|city_name|other)/(ad_pos/)?)?
        page_type=1
        fields=path.split('/')
        cate_name=fields[1]
        unknown=fields[5]
        city_name,ad_id,ad_pos='',-1,-1
        if unknown not in self.detail_others:
            try:
                long(unknown)
                ad_id,ad_pos=unknown,fields[6]
            except Exception:
                city_name=unknown
        return page_type,city_name,cate_name,ad_id,ad_pos
        
    def reduce(self):
        city_dict=self.city_dict
        cate_dict=self.cate_dict
        tels={}
        for fields in self.map():
            fields.pop(0)
            page_type=fields[0]
            if page_type==0:
                city1,city2=city_dict['id'].get(long(fields[1]),('',''))
            elif page_type==1:
                city1,city2=city_dict['name'].get(fields[1],('',''))
            cate_name,ad_id,ad_pos=fields[2:]
            cate1,cate2,cate3=cate_dict.get(cate_name,('','',''))
            key=(page_type,city1,city2,cate1,cate2,cate3,ad_pos)
            tels.setdefault(key,1)
            tels[key]+=1
        for key in tels:
            yield [self.date]+list(key)+[tels[key]]
            
    @property
    def table_fields(self):
        fields=[
            "stat_date date comment '统计日期'",
            "page_type tinyint(2) comment '页面类型 0:列表页 1:详情页'",
            "city1 varchar(30) comment '一级城市'",
            "city2 varchar(30) comment '二级城市'",
            "cate1 varchar(30) comment '一级类别'",
            "cate2 varchar(30) comment '二级类别'",
            "cate3 varchar(30) comment '三级类别'",
            "ad_pos int(10) comment '广告位置 没有则为-1'",
            "tel_count int(10) comment '电话数'",
            "primary key (`stat_date`,`page_type`,`city1`,`city2`,`cate1`,`cate2`,`cate3`,`ad_pos`)",
        ]
        return fields
    
    @property
    def table(self):
        return 'mlog_telphone'
    
    def load(self):
        table=self.table
        dbi=DBI(**dbc[self.db])
        dbi.create(table,self.table_fields,options=dict(ENGINE='InnoDB'))
        dbi.delete(table,conditions=dict(stat_date=self.date))
        if self.output=='.':
            path=os.getcwd()
            reduce_file=path+self.reduce_file.split('.',1)[1]
        else:
            reduce_file=self.reduce_file
        reduce_file=repr(reduce_file).strip("'")
        logger.info("Load file '%s' into table '%s'" % (reduce_file,table))
        dbi.load(reduce_file,table,field_sep=',')
        dbi.close()
        
@looper('DAY')
def main(date):
    input='/work2/opt/analytic/precise/mlog/log'
    output='/work2/opt/analytic/precise/mlog/file'
    mt=MlogTelphone(date,input,output)
    mt.get_logs()
    mt.store()
    mt.load()
    
if __name__ == '__main__':
    main()
